#!/usr/bin/env python
# coding=utf-8
'''
Created on 2010-7-27

@author: Jason.q.yan
'''
import os, sys, re, traceback, logging
import urllib
import time
import datetime
import getopt
import math
import urllib2
currpath=sys.path[0]
root_path=currpath[:currpath.find('src')]
print root_path+'src/tools'
sys.path.append(root_path+'src')
import MySQLdb,setting

pageNo_re = re.compile(r'''
        <span>页次：\d+/(\d+)页\s共\d+条</span>
        ''', re.X|re.S)
routeDiv_re = re.compile(r'''
        <li>\s+<a\s+href([\s\S]+?)</li>
        ''', re.X|re.S)

url_re = re.compile(u'''
        "/qybl/compinfo_(\d+?).html"
        ''', re.X|re.S)
company_name_re = re.compile(u'''
        target="_blank">([\S]+?)</a><span>([\s\S]+?)</span></dt>
        ''', re.X|re.S)
region_re = re.compile(u'''
        target="_blank">(\S+?)</a></b></dd>
        ''', re.X|re.S)
products_re = re.compile(r'''
        主要产品：([\s\S]+?)</dd>
        ''', re.X|re.S)
adept_re = re.compile(r'''
                     主要优势：([\s\S]+?)</dd>
        ''', re.X|re.S)
telphone_re = re.compile(r'''
                    电&nbsp;&nbsp;话：<span\stitle="([\s\S]+?)">
        ''', re.X|re.S)
telex_re = re.compile(r'''
                    传&nbsp;&nbsp;真：<span\stitle="([\s\S]+?)">
        ''', re.X|re.S)
messager_re = re.compile(u'''
        target="_blank">(\S+?)</a></b>&nbsp;
        ''', re.X|re.S)
honesty_year_re = re.compile(r'''
        title="诚信录会员第(\d+)年"
        ''', re.X|re.S)
honesty_exp_re = re.compile(r'''
        target="_blank">(\d+?)</a>
        ''', re.X|re.S)

server_urls = "http://www.17u.net/qybl/list_0_0_1_0_2_%s.html"


insert_sql = "insert into SUP_MESSAGER(url,company_name,region,products,adept,telphone,telex,messager,honesty_year,honesty_exp) values('%s','%s','%s','%s','%s','%s','%s','%s','%s','%s')"
check_sql = "select url from SUP_MESSAGER where url = '%s'"

def do_crawl():
    conn = MySQLdb.Connection(setting.server_vacation,
                              setting.username_vacation,
                              setting.password_vacation,
                              setting.database_vacationdb,
                              use_unicode=True,
                              charset='utf8')
    curs=conn.cursor()
    
    html = urllib2.urlopen(server_urls % '1').read().decode('gbk','ignore').encode('utf-8')
    page_no = pageNo_re.search(html).group(1)
    for i in range(1,int(page_no)):
        try:
            html = urllib2.urlopen(server_urls % str(i)).read().decode('gbk','ignore').encode('utf-8')
            div_list = routeDiv_re.findall(html)
            for div in div_list:
                try:
                    url = "http://www.17u.net/qybl/compinfo_" + url_re.search(div).group(1) + ".html"
                    #if curs.execute(check_sql % url) > 0:
                    #    print >>sys.stderr, 'existed', url
                    #    continue
                    company_name = company_name_re.search(div).group(1) + "(" + company_name_re.search(div).group(2) + ")"
                    region = region_re.search(div).group(1)
                    products = products_re.search(div).group(1)
                    adept = adept_re.search(div).group(1)
                    telphone = telphone_re.search(div).group(1)
                    telex = telex_re.search(div).group(1)
                    messager = messager_re.search(div).group(1)
                    honesty_year = honesty_year_re.search(div).group(1)
                    honesty_exp = honesty_exp_re.search(div).group(1)
                    curs.execute(insert_sql % (url,company_name,region,products,adept,telphone,telex,messager,honesty_year,honesty_exp))
                    print >>sys.stderr, 'insert', url
                except:
                    print >>sys.stderr, 'error', url
                continue
        except:
            print >>sys.stderr, 'error page:', i
            continue
        

if __name__ == '__main__':
    script_path = os.path.dirname(os.path.realpath(__file__))
    os.chdir(script_path)
    do_debug = False

    try:
        opts, args = getopt.gnu_getopt(sys.argv[1:], 'hdno:p:', ['help', 'debug', 'no-cache', 'only-cache', 'provider'])
    except getopt.GetoptError:
        usage()
        sys.exit()

    for opt, arg in opts:
        if opt in ('-h', '--help'):
            usage()
            sys.exit()
        elif opt in ('-d', '--debug'):
            do_debug = True
        elif opt in ('-n', '--no-cache'):
            use_cache = False
        elif opt in ('-o', '--only-cache'):
            if arg.lower() in ('no', 'n', '0'):
                only_cache = False
            else:
                only_cache = True
        elif opt in ('-p', '--provider'):
            pass

    if do_debug:
        import pdb
        pdb.set_trace()
    do_crawl()
